{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for RentListingInquries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 用GridSearchCV调整max_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导入准备调用的模块\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取train的数据文件并显示头5行数据\n",
    "train = pd.read_csv(\"./data/RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#分离特征列与目标列\n",
    "y_train = train['interest_level']\n",
    "\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#从前序代码的countplot图可知，三类目标的样本数量不均匀，故采用分层采样，考虑时间代价，将划分等级设为3，可能会影响最终的模型参数\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(5, 10)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#设定max_depth调整的范围\n",
    "max_depth = range(5,10)\n",
    "param_t1=dict(max_depth=max_depth)\n",
    "param_t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#实例化max_depth调优的XGBClassifier\n",
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #这里代入的是第一轮参数调整得到的n_estimators值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59284, std: 0.00295, params: {'max_depth': 5},\n",
       "  mean: -0.59335, std: 0.00446, params: {'max_depth': 6},\n",
       "  mean: -0.59515, std: 0.00271, params: {'max_depth': 7},\n",
       "  mean: -0.60356, std: 0.00209, params: {'max_depth': 8},\n",
       "  mean: -0.61345, std: 0.00317, params: {'max_depth': 9}],\n",
       " {'max_depth': 5},\n",
       " -0.59283640164901674)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#实例化max_depth调优的GridSearchCV,并将以上的调优取值集合与学习器代入\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_t1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train) #利用实例好的GridSearchCV来训练数据\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_ #显示cv后的结果与最佳参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上可知，计算出的max_depth最优值为5，但原设定的取值范围为range(5,10)，最优值在其下限，故需重新设定max_depth下限范围再做一轮参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6]}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#重新设定max_depth参数调节范围\n",
    "max_depth = [4,5,6]\n",
    "param_t2=dict(max_depth=max_depth)\n",
    "param_t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59634, std: 0.00356, params: {'max_depth': 4},\n",
       "  mean: -0.59284, std: 0.00295, params: {'max_depth': 5},\n",
       "  mean: -0.59335, std: 0.00446, params: {'max_depth': 6}],\n",
       " {'max_depth': 5},\n",
       " -0.59283640164901674)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#实例化max_depth调优的GridSearchCV,并将以上的调优取值集合与学习器代入\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_t2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train) #利用实例好的GridSearchCV来训练数据\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从结果来看，将max_depth取值的下限范围重新设定后，其最优值仍为5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parameters</th>\n",
       "      <th>mean_validation_score</th>\n",
       "      <th>cv_validation_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'max_depth': 4}</td>\n",
       "      <td>-0.596341</td>\n",
       "      <td>[-0.591308677575, -0.59906909175, -0.598646431...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'max_depth': 5}</td>\n",
       "      <td>-0.592836</td>\n",
       "      <td>[-0.588689694359, -0.594463642867, -0.59535602...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'max_depth': 6}</td>\n",
       "      <td>-0.593354</td>\n",
       "      <td>[-0.587258624833, -0.594994522235, -0.59780918...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         parameters  mean_validation_score  \\\n",
       "0  {'max_depth': 4}              -0.596341   \n",
       "1  {'max_depth': 5}              -0.592836   \n",
       "2  {'max_depth': 6}              -0.593354   \n",
       "\n",
       "                                cv_validation_scores  \n",
       "0  [-0.591308677575, -0.59906909175, -0.598646431...  \n",
       "1  [-0.588689694359, -0.594463642867, -0.59535602...  \n",
       "2  [-0.587258624833, -0.594994522235, -0.59780918...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gsearch2_1.grid_scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>213.958905</td>\n",
       "      <td>1.294407</td>\n",
       "      <td>-0.596341</td>\n",
       "      <td>-0.545236</td>\n",
       "      <td>4</td>\n",
       "      <td>{'max_depth': 4}</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.591309</td>\n",
       "      <td>-0.547863</td>\n",
       "      <td>-0.599069</td>\n",
       "      <td>-0.543747</td>\n",
       "      <td>-0.598646</td>\n",
       "      <td>-0.544099</td>\n",
       "      <td>25.742679</td>\n",
       "      <td>0.370765</td>\n",
       "      <td>0.003563</td>\n",
       "      <td>0.001863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>223.333774</td>\n",
       "      <td>1.116397</td>\n",
       "      <td>-0.592836</td>\n",
       "      <td>-0.502762</td>\n",
       "      <td>5</td>\n",
       "      <td>{'max_depth': 5}</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.588690</td>\n",
       "      <td>-0.505629</td>\n",
       "      <td>-0.594464</td>\n",
       "      <td>-0.501737</td>\n",
       "      <td>-0.595356</td>\n",
       "      <td>-0.500920</td>\n",
       "      <td>7.568460</td>\n",
       "      <td>0.040715</td>\n",
       "      <td>0.002955</td>\n",
       "      <td>0.002055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>224.674517</td>\n",
       "      <td>1.192068</td>\n",
       "      <td>-0.593354</td>\n",
       "      <td>-0.452357</td>\n",
       "      <td>6</td>\n",
       "      <td>{'max_depth': 6}</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.587259</td>\n",
       "      <td>-0.453763</td>\n",
       "      <td>-0.594995</td>\n",
       "      <td>-0.451512</td>\n",
       "      <td>-0.597809</td>\n",
       "      <td>-0.451795</td>\n",
       "      <td>30.164409</td>\n",
       "      <td>0.104985</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.001001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "0     213.958905         1.294407        -0.596341         -0.545236   \n",
       "1     223.333774         1.116397        -0.592836         -0.502762   \n",
       "2     224.674517         1.192068        -0.593354         -0.452357   \n",
       "\n",
       "  param_max_depth            params  rank_test_score  split0_test_score  \\\n",
       "0               4  {'max_depth': 4}                3          -0.591309   \n",
       "1               5  {'max_depth': 5}                1          -0.588690   \n",
       "2               6  {'max_depth': 6}                2          -0.587259   \n",
       "\n",
       "   split0_train_score  split1_test_score  split1_train_score  \\\n",
       "0           -0.547863          -0.599069           -0.543747   \n",
       "1           -0.505629          -0.594464           -0.501737   \n",
       "2           -0.453763          -0.594995           -0.451512   \n",
       "\n",
       "   split2_test_score  split2_train_score  std_fit_time  std_score_time  \\\n",
       "0          -0.598646           -0.544099     25.742679        0.370765   \n",
       "1          -0.595356           -0.500920      7.568460        0.040715   \n",
       "2          -0.597809           -0.451795     30.164409        0.104985   \n",
       "\n",
       "   std_test_score  std_train_score  \n",
       "0        0.003563         0.001863  \n",
       "1        0.002955         0.002055  \n",
       "2        0.004461         0.001001  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gsearch2_1.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.592836 using {'max_depth': 5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8lOXV//HPSUgIm7KqCChQgQKC\nAQKiFgTrAkJR6wZKH6FVWpWi9pEH+fWxrdQ+3VFR1GprVUBRsCoqKrgvLBIkbAFk0UrEagAXdgg5\nvz/uO2ESkkwgM5ks3/frNS9m7vXkzmQO13Xmvi5zd0RERMqSlOgARESk6lOyEBGRqJQsREQkKiUL\nERGJSslCRESiUrIQEZGolCxERCQqJQsREYlKyUJERKKqk+gAYqV58+betm3bRIchIlKtLF26dKu7\nt4i2XY1JFm3btiUzMzPRYYiIVCtm9u/ybBfXbigzG2Rm68xsg5ndVso2V5hZtpmtNrMnIpZfY2br\nw8c18YxTRETKFreWhZklA1OB84AcYImZzXH37IhtOgATgbPc/SszOy5c3hT4NZABOLA03PereMUr\nIiKli2fLog+wwd03uft+YCZwUbFtrgOmFiQBd/8yXH4BMN/dt4fr5gOD4hiriIiUIZ7JohWwOeJ1\nTrgsUkego5m9b2aLzGzQEeyLmY0xs0wzy8zNzY1h6CIiEimeycJKWFZ88ow6QAdgADAC+LuZNS7n\nvrj7Q+6e4e4ZLVpELeaLiMhRimeyyAHaRLxuDWwpYZvn3f2Au38MrCNIHuXZV0REKkk8k8USoIOZ\ntTOzVGA4MKfYNs8BAwHMrDlBt9Qm4FXgfDNrYmZNgPPDZSIikgBx+zaUu+eZ2ViCD/lk4BF3X21m\nk4BMd5/DoaSQDRwExrv7NgAz+y1BwgGY5O7b4xHn3gMHeWrJZurWSaJuShJ16yQHz+skk1bwOiWp\ncFnkdslJJfWWiYjUPFZT5uDOyMjwo7kpL3fHPnr/7rWjOmedJKNunSTSUgqSSEGiKTvJFE1KJe+X\nVtb+4bZmSlYiUjFmttTdM6JtV2Pu4D5azRqk8uHt57H3wEH25eWzL+8g+w7kl/w8L599Bw6yNy8/\nXF7KPuH6nfvy2LYzcrtg/4LnFZUaJo20qEkqOdyuaMuptGRUZLtSEludJFOyEqlFan2ySEoymjZI\nrfTzujv7D+YXJpYiyeqwZHQoyeyNSDalJamC51/v3l9iktqXd5ADByvWokwyytVVV7d4QitH6yvt\nsNbX4dslqQtQpFLV+mSRKGYWfvAlQ1rln/9gvrM/TBx7D5SWpA5PRnsPlJ2gIvf7Zs+BErfdm3eQ\nivZ+piTbYUkltYxkdKj1VXaLqcxWWvg8NVldgFWBu5Pvwb8OuIPjwb+Rz4F8D54TsTy/2H44wfEi\n9nMv+XgF56bIskPnKdgWIpdFxlp0v8KfJYyjyPEi9sNLPt6x9VLo275ZXK+3kkUtlZxk1EtNpl5q\ncqWf293Jy/di3Xold9UVT1ilJ7bI/Q6yY28eW/P2l5jw9lewC9CMshNPSS0hs2IfNsU+OCj64XDY\nh1rEflDSh2TkB9ChD7hSjxexHxEfVIc+GEs4HsG+OCXH7Yd+v5Ef0GXGXeIHafS4paj0No157saz\n4noOJQupdGZGSrKRkpxEw7qV/xbMz4/oAixMJgWJqBzJqBxdgdt37S+yf747hmEW3HFqFj43Cpcn\nmQV3o0ZskxSxnshlEcuLHK/gOcHxCo+VBEYSSUmlHC9ivyCuiOcR8VHCtkXjLtg2Ir5wZdlxH1qW\nlBTleEHghy0rjLv48azozxssK3rNyjzeYdcmvBalXIeCn7f4siJxFbu+SVb05428DqUeL+L61k+N\n/9+RkoXUOklJRlpSMmkpyUBKosMRqRY0U56IiESlZCEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIi\nUSlZiIhIVEoWIiISlZKFiIhEpWQhIiJRKVmIiEhUShYiIhJVXJOFmQ0ys3VmtsHMbith/SgzyzWz\nrPBxbcS6P5nZajNbY2ZTTBMIiIgkTNxGnTWzZGAqcB6QAywxsznunl1s06fcfWyxfc8EzgK6h4ve\nA84G3opXvCIiUrp4tiz6ABvcfZO77wdmAheVc18nmD8uFahLMI70F3GJUkREoopnsmgFbI54nRMu\nK+5SM1thZrPNrA2Auy8E3gQ+Dx+vuvuaOMYqIiJliGeyKKnGUHxCxBeAtu7eHXgNeAzAzE4BOgOt\nCRLMOWbW/7ATmI0xs0wzy8zNzY1p8CIickg8k0UO0CbidWtgS+QG7r7N3feFLx8GeoXPLwEWuftO\nd98JvAz0LX4Cd3/I3TPcPaNFixYx/wFERCQQz2SxBOhgZu3MLBUYDsyJ3MDMWka8HAYUdDV9Cpxt\nZnXMLIWguK1uKBGRBInbt6HcPc/MxgKvAsnAI+6+2swmAZnuPgcYZ2bDgDxgOzAq3H02cA6wkqDr\n6hV3fyFesYqISNnMvXgZoXrKyMjwzMzMRIchIlKtmNlSd8+Itp3u4BYRkaiULEREJColCxERiUrJ\nQkREolKyEBGRqJQsREQkKiULERGJSslCRESiUrIQEZGolCxERCQqJQsREYlKyUJERKJSshARkaiU\nLEREJColCxERiUrJQkREolKyEBGRqJQsREQkqrgmCzMbZGbrzGyDmd1WwvpRZpZrZlnh49qIdSeZ\n2TwzW2Nm2WbWNp6xiohI6erE68BmlgxMBc4DcoAlZjbH3bOLbfqUu48t4RCPA79z9/lm1hDIj1es\nIiJStni2LPoAG9x9k7vvB2YCF5VnRzPrAtRx9/kA7r7T3XfHL1QRESlLPJNFK2BzxOuccFlxl5rZ\nCjObbWZtwmUdga/N7F9mtszM/hy2VEREJAHimSyshGVe7PULQFt37w68BjwWLq8D9ANuBXoD7YFR\nh53AbIyZZZpZZm5ubqziFhGRYuKZLHKANhGvWwNbIjdw923uvi98+TDQK2LfZWEXVh7wHNCz+Anc\n/SF3z3D3jBYtWsT8BxARkUA8k8USoIOZtTOzVGA4MCdyAzNrGfFyGLAmYt8mZlaQAc4BihfGRUSk\nksTt21DunmdmY4FXgWTgEXdfbWaTgEx3nwOMM7NhQB6wnbCryd0PmtmtwOtmZsBSgpaHiIgkgLkX\nLyNUTxkZGZ6ZmZnoMEREqhUzW+ruGdG20x3cIiISlZKFiIhEpWQhIiJRKVmIiEhUShYiIhKVkoWI\niESlZCEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhEpWQB8NE82L8r\n0VGIiFRZcZv8qNrYvgmeuBxSG8GpP4QeP4LWGWAlTSEuIlI7qWXRpB2MfgW6XAQrZ8M/zoWpp8P7\nU2Dnl4mOTkSkStBMeZH27YDVz8GyabB5MSTVgQ4XQI+R0OF8SFZDTERqlvLOlKdPv0h1G0HPHwWP\n3I8gazpkPQnrXoKGx8NpwyF9JLTomOhIRUQqVVy7ocxskJmtM7MNZnZbCetHmVmumWWFj2uLrT/G\nzD4zs/viGWeJWnSE8ybBL7JhxExo3RsWToWpveEf58OHjwctERGRWiBu3VBmlgx8BJwH5ABLgBHu\nnh2xzSggw93HlnKMe4AWwPbStikQk26oaHZ+CSuegg+nwdZ1kFIful4SdFOddIaK4iJS7VSFbqg+\nwAZ33xQGNBO4CMguc6+QmfUCjgdeAaL+IJWi4XFw5s/hjLGQkxnUNlb9C7JmQNP2QdI4bQQcc2Ki\nIxURial4dkO1AjZHvM4JlxV3qZmtMLPZZtYGwMySgL8C4+MY39Ezgza9YdgUuHUdXPwgNDoRXp8E\nd3WFGZdD9vOQtz/RkYqIxEQ8WxYl9ckU7/N6AXjS3feZ2c+Ax4BzgBuAue6+2cro2jGzMcAYgJNO\nOikmQR+x1AaQPiJ4bNsIWU8Ej6f/C+o3g+7DgxbH8V0SE5+ISAxErVmY2XeAnPADfQDQHXjc3b+O\nst8ZwG/c/YLw9UQAd/99KdsnE9QmjjWzGUA/IB9oCKQC97v7YUXyApVSsyiv/IOw8c2gm2rtS5B/\nAE7sGSSNUy+Feo0THaGICFD+mkV5kkUWQc2gLfAqMAfo5O4XRtmvDkGB+/vAZwQF7qvcfXXENi3d\n/fPw+SXABHfvW+w4oyijCF6gSiWLSLu2wcqng6L4l6uhThp0HhYkjrb9IEn3RYpI4sSywJ3v7nnh\nh/nd7n6vmS2LtlO4z1iCBJMMPOLuq81sEpDp7nOAcWY2DMgDtgOjyhFP9dKgGfS9Hk7/GXyeBcum\nw4pZQQJpfPKhonjjNomOVESkVOVpWSwG7gZ+CfzA3T82s1XufmplBFheVbZlUZIDe4LuqWXTYNNb\ngMF3BgaJo9MQSElLdIQiUkvEsmUxGvgZ8LswUbQDplc0wFotpR50uyx4fPXvsCg+A2b/GNIaQ/cr\ngsTR8rRERyoiAhzhTXlm1gRo4+4r4hfS0alWLYuS5OfDx28H3VRrXoCD++CEbsEouN0uh/pNEx2h\niNRAsSxwvwUMI2iFZAG5wNvu/osYxBkz1T5ZRNrzVTAC7rLpQZ0jORW+OyRobbQfCEnJiY5QRGqI\nWHZDHevu34bjNv3T3X9tZlWuZVGj1GsCfa4LHv9ZCctmBMOMrH4WjmkN6VcFj6btEh2piNQS5fne\nZh0zawlcAbwY53ikuBO6weA/wH+vhcsfg+M6w7t/gSnp8OhQWP4U7N+d6ChFpIYrT8tiEsHXX993\n9yVm1h5YH9+w5DB16kLXi4PHN5/B8ieCbqpnx8DcY4Kb/Xr8CFr11ICGIhJzmvyoOsvPh08XBElj\n9XOQtwdadA5qG92vhIYtEh2hiFRxsSxwtwbuBc4iGNvpPeAmd8+JRaCxUiuTRaS938LqfwWJI2dJ\nMMtfx0FBa+OUczXLn4iUKJYF7n8CTwCXh69HhsvOO/rwJObSjoFeo4LHl2uDWf6Wz4S1L0LDE8LB\nDkdC81MSHamIVEPlGhvK3dOjLUu0Wt+yKMnBA/DRq0FrY/088IPBJE09RkKXi6Fuw0RHKCIJVt6W\nRXm+DbXVzEaaWXL4GAlsq3iIEnfJKdB5KFw1M5ge9tw7YNdWeP5G+EvH4N9PF0ENqVuJSPyUp2Vx\nEnAfcAZBzWIBMM7dP41/eOWnlkU5ucPmD4JxqVY/C/t3QrNTDg1o2OiEREcoIpUoZgXuUg5+s7vf\nfVSRxYmSxVHYtxOynwu6qT5dCJYMHc4PEkfHC4KWiYjUaPFOFp+6e4KmpiuZkkUFbd0QFMWznoSd\n/4H6zeG0cJa/4zonOjoRiZN4J4vN7l6lJmBQsoiRg3mw8fWgm2rdy5CfB60ywln+fghpxyY6QhGJ\nIbUspOJ2bQ3GpPpwGuSugTr1oMtFQeI4+SzN8idSA1Q4WZjZDoKC9mGrgHruXqXu8lKyiCN32PJh\nUNtYORv2fQtN2gb3baRfBce2SnSEInKU4tqyqIqULCrJ/t3BfBvLpsEn7wIGp3w/nOXvwmAMKxGp\nNmJ5n0VFghhkZuvMbIOZ3VbC+lFmlmtmWeHj2nB5upktNLPVZrbCzK6MZ5xyBFLrw2lXwqgXYVwW\n9B8f3DE+axT8tRO8PCEYVl1EapS4tSzMLBn4iGBYkBxgCTDC3bMjthkFZLj72GL7dgTc3deb2YnA\nUqCzu39d2vnUskig/IPBXOLLpgfDixzcH0wJ2+NHwdSx9ZokOkIRKUVVaFn0ATa4+yZ33w/MBC4q\nz47u/pG7rw+fbwG+BDSEalWVlBx0RV3+T/jvdTD4z+D5MPdW+EunYG7xjW8Eo+SKSLUUzyJ1K2Bz\nxOsc4PQStrvUzPoTtEJucffIfTCzPkAqsDFegUoM1W8Kp48JHp8vD1obK56GVc/AsW3CWf6uhiYn\nJzpSETkCUVsWZrbDzL4t9thsZs+GEyGVumsJy4r3eb0AtHX37sBrwGPFzt0SmAaMdvfD/ltqZmPM\nLNPMMnNzc6P9KFLZWp4GF/45aG1c9gg07wBv/wnu6Q6PDYMVs+DAnkRHKSLlUJ6xoe4AthAMU27A\ncOAEYB1wvbsPKGW/M4DfuPsF4euJAO7++1K2Twa2u/ux4etjgLeA37v7rGg/iGoW1cTXm2H5k8G3\nqb7+FOoeG9Q1eoyEE3tolj+RShbLyY8Wu/vpxZYtcve+Zrbc3U8rZb86BF1L3wc+IyhwX+XuqyO2\naenun4fPLwEmhMdNBV4GXijvGFRKFtVMfj78+73ghr81cyBvLxzX9dAsfw2aJTpCkVohlgXufDO7\nwsySwscVEetKzTTungeMJZi/ew3wtLuvNrNJZjYs3Gxc+PXY5cA4YFS4/AqgPzAq4mu1VWr+DKmg\npCRo1x8ufTjophoyObhH49WJwVdwn/4vWD8/+KaViCRceVoW7YF7CIYoB1gI3ELQWujl7u/FNcJy\nUsuihvhiNSybAStmwu5t0OjEcJa/q6HZdxIdnUiNozu4pXrL2w8fvRJ8m2rD/OCruCefFc7ydxGk\nNkh0hCI1QixrFq2Be4GzCLqd3gNucvecWAQaK0oWNdi3W4L5xJdNh+0bIbVhMAJujx9B694qiotU\nQCyTxXyCb0JNCxeNBK529/MqHGUMKVnUAu7BNLAFs/wd2A3NO4ZF8eHQ6PhERyhS7cQyWWS5e3q0\nZYmmZFHL7NsRJIxl02Hz4mCWv46DgsTR4TzN8idSTuVNFuW5g3urmY0EngxfjwC2VSQ4kQqr2wh6\n/lfwyP3o0Cx/616CBscdmuWvRadERypSI5SnZXEScB/Bt6EcWACMc/dP4x9e+allIRw8ABteC1ob\nH70SzPLXuk+QNLpeAmnHJDpCkSon3jPl3Vzem+Uqi5KFFLHzy0NF8a3rIKU+dLk4nOXvTBXFRUKa\nVlUEgqJ4TmZQFF/1L9i/A5q2D+7bSL8Kjjkx0RGKJFS8k8Vmd29zVJHFiZKFRLV/F2TPCVob/34P\nLAlOOTdobXQcDHVSEx2hSKWLZYG7JDXjTj6pXVIbhHeDj4BtGyFrBmQ9EQwtUr9ZMCZVj5FwfNdE\nRypS5ZTasjCzHZScFAyo5+7xnAvjiKllIUcl/2AwMdOyabB2LuQfCEa/7TESTr0M6jVOdIQicaXh\nPkSO1K5tsPLpYCTcL1dDnTTo/IPgTvG2/YLBD0VqGCULkaPlDp9nhbP8zYJ930DjkyB9ZNCF1bhK\nfbdDpEKULERi4cAeWPNi0E318duAQfsBQTfVd4dCSlqCAxSpmHgXuEVqh5R60P3y4PHVv4OCeNYM\neOYnkHYsdLsiSBwtT9O9G1KjqWUhcqTy84NWxrLpsOYFOLgPju8WDmh4BdRvmugIRcpN3VAilWHP\nV7BydpA4Ps+C5FTodCH0/BG0HwhJyYmOUKRMShYile0/Kw/N8rfnKzimVXCXePpVwV3jIlWQkoVI\nouTtg3Vzw1n+XgccvnMOnDkuKI6rtiFVSHmTRVy/OG5mg8xsnZltMLPbSlg/ysxyzSwrfFwbse4a\nM1sfPq6JZ5wiMVWnbjDK7chn4JZVMPCX8J9VMO1i+Fs/WPF0MEKuSDUSt5aFmSUDHwHnATnAEmCE\nu2dHbDMKyHD3scX2bQpkAhkEd5EvBXq5+1elnU8tC6nSDuwNbvhbcC9s/QiOaQ19rw/m49DQ6ZJA\nVaFl0QfY4O6b3H0/MBO4qJz7XgDMd/ftYYKYDwyKU5wi8ZeSFiSGGxbDiKegSVuY90u4qyvMuz2Y\nZ1ykCotnsmgFbI54nRMuK+5SM1thZrPNrGAk2/LuK1K9JCVBp0Ew+iW47g045fuw8D64uxs8+zP4\nYnWiIxQpUTyTRUlVvOJ9Xi8Abd29O/Aa8NgR7IuZjTGzTDPLzM3NrVCwIpWuVS+4/FEYtwwyfgLZ\nz8MDZ8K0H8LGN4NhR0SqiHgmixwgcs6L1kCRtra7b3P3feHLh4Fe5d033P8hd89w94wWLVrELHCR\nStWkLVz4J7hlNZxze/AVXBXDpYqJZ7JYAnQws3ZmlgoMB+ZEbmBmLSNeDgPWhM9fBc43syZm1gQ4\nP1wmUnPVbwr9b4WbV8Kwe4Ov4P7rOrgnHRbcB3u/TXSEUovFLVm4ex4wluBDfg3wtLuvNrNJZjYs\n3Gycma02s+XAOGBUuO924LcECWcJMClcJlLzlVoMPxXm/0rFcEkI3ZQnUh3kLIWF9wZ1DUuGbpfD\nmWM1q59UmEadFalJWofF8O0fw6IHgiHTlz8RzCF+5s+h3dm6M1ziSlN/iVQnTdtFFMP/Fz5fAY9f\nBH/rH0zUpGK4xImShUh1VL8p9B8fFMN/MAXy9sK/roUpPWDhVNi3I9ERSg2jZCFSnaWkQa9rwmL4\nzGDK11f/H0zuqmK4xJSShUhNkJQEnQbD6Llw7RvwnYHBOFR3d4dnr4cvsqMfQ6QMKnCL1DSte8EV\nj4XF8PuDodJVDJcKUstCpKZq2g4u/HNEMXy5iuFy1JQsRGq6wmL4qqAYfmCPiuFyxJQsRGqLgmL4\njR+UUAz/NXz7eaIjlCpMyUKktilSDH89LIZPCYZJf+4GFcOlRCpwi9RmrTPCYvim8M7w6ZA1IyyG\nj4N2/VUMF0AtCxEBaNr+UDF8YEExfBg8dDasnK1iuChZiEiE+k3h7IJi+D2wfzc88xMVw0XJQkRK\nkJIGvUYdKoYf20bF8FpOyUJESldQDP/xy2ExfICK4bWUCtwiUj6tM+CKxw8Vwz+cFhbDzwvvDFcx\nvCZTy0JEjkxBMfwX2WExPKtYMTwv0RFKHChZiMjRKbUYng4L71cxvIaJa7Iws0Fmts7MNpjZbWVs\nd5mZuZllhK9TzOwxM1tpZmvMbGI84xSRCogshg9/MiyGT4S7usJrv1ExvIaIW7Iws2RgKjAY6AKM\nMLMuJWzXCBgHLI5YfDlQ1927Ab2An5pZ23jFKiIxkJQE373wUDG8/QB4/56wGH4jfLkm0RFKBcSz\nZdEH2ODum9x9PzATuKiE7X4L/AnYG7HMgQZmVgeoB+wHvo1jrCISSwXF8J8vDVodq56B+/vC9Mvg\n43fAPdERyhGKZ7JoBWyOeJ0TLitkZj2ANu7+YrF9ZwO7gM+BT4G/uPv2OMYqIvHQtD0M+UtYDP8l\nbFkGj/1AxfBqKJ7JoqTv0BX+d8LMkoC7gP8uYbs+wEHgRKAd8N9m1v6wE5iNMbNMM8vMzc2NTdQi\nEnv1m8LZ/wO3rIKhd8P+XRF3hqsYXh3EM1nkAG0iXrcGIicEbgScCrxlZp8AfYE5YZH7KuAVdz/g\n7l8C7wMZxU/g7g+5e4a7Z7Ro0SJOP4aIxExKPcgYDTcuCYvhrYoWw3f8J9ERSinimSyWAB3MrJ2Z\npQLDgTkFK939G3dv7u5t3b0tsAgY5u6ZBF1P51igAUEiWRvHWEWkMhUWw1+Bn7wWTPX63t1w16kq\nhldRcUsW7p4HjAVeBdYAT7v7ajObZGbDouw+FWgIrCJIOv909xXxilVEEqhNb7hyGoz7sGgxfMbl\nKoZXIeY15BeRkZHhmZmZiQ5DRCpq1zbI/Acs/hvs3got04PhRLpcDMkaoSjWzGypux/WzV+c7uAW\nkaqlQbNixfCdh4rhix6AfTsTHWGtpGQhIlVTkWL4E0Ex/JXb4K4u8NodKoZXMiULEanakpLgu0OK\nFcPvirgzXN99qQw1ugPwwIED5OTksHfv3ugbS7WSlpZG69atSUlJSXQoUpkKiuHbNsKi+2HZDMia\nDh3OD+YMb/s9DZMeJzW6wP3xxx/TqFEjmjVrhukNVGO4O9u2bWPHjh20a9cu0eFIIu3aBkv+Dh88\npGL4UVKBG9i7d68SRQ1kZjRr1kwtRgmK4QMmqBheCWp0sgCUKGoo/V6liMhi+JUz4JgTVQyPsRqf\nLESkFklKgs5D4Sevwk/mB1O9FhTDn1cxvCKULOLs66+/5v777z+qfe+++252794d44hiY8CAARzt\nTZDPPfcc2dnZMTmWSKna9IErpwfDpPf4UTDK7f2nw4wr4ON3dWf4EVKyiLOamiwqoniyEImrZt+B\noZPhltUwYCJ8lgmPDYWHBwZDi2iY9HJRsoiz2267jY0bN5Kens748eP585//TO/evenevTu//vWv\nAdi1axdDhgzhtNNO49RTT+Wpp55iypQpbNmyhYEDBzJw4MBSj9+wYUMmTJhAr169OPfcc/nggw8Y\nMGAA7du3Z86cYNzGTz75hH79+tGzZ0969uzJggULAHj22Wc599xzcXc+//xzOnbsyH/+U3Lf7p49\nexg+fDjdu3fnyiuvZM+ePYXr5s2bxxlnnEHPnj25/PLL2bkzKCq2bduWCRMm0KdPH/r06cOGDRtY\nsGABc+bMYfz48aSnp7Nx40YAZs2aRZ8+fejYsSPvvvtuxS+8SHENmsOA24KkMfQu2PstzP4x3NsD\nFj2oYngUtea7ZXe8sJrsLbGdbK/Licfw6x90LXObP/zhD6xatYqsrCzmzZvH7Nmz+eCDD3B3hg0b\nxjvvvENubi4nnngiL730EgDffPMNxx57LJMnT+bNN9+kefPmpR5/165dDBgwgD/+8Y9ccskl/O//\n/i/z588nOzuba665hmHDhnHccccxf/580tLSWL9+PSNGjCAzM5NLLrmEZ555hqlTp/LKK69wxx13\ncMIJJ5R4ngceeID69euzYsUKVqxYQc+ePQHYunUrd955J6+99hoNGjTgj3/8I5MnT+ZXv/oVAMcc\ncwwffPABjz/+ODfffDMvvvgiw4YNY+jQoVx22WWFx8/Ly+ODDz5g7ty53HHHHbz22mtH9LsQKbeU\nepDxY+h5Dax7GRZMgVcmwFu/h94/gT4/hUbHJzrKKqfWJIuqYN68ecybN48ePXoAsHPnTtavX0+/\nfv249dZbmTBhAkOHDqVfv37lPmZqaiqDBg0CoFu3btStW5eUlBS6devGJ598AgQ3J44dO5asrCyS\nk5P56KOPCve/9957OfXUU+nbty8jRowo9TzvvPMO48aNA6B79+50794dgEWLFpGdnc1ZZ50FwP79\n+znjjDMK9ys45ogRI7jllltKPf4Pf/hDAHr16lUYt0hcJSUHxfDOQ+HTxbDwXnh3Miy4F7pfGdyv\n0aJToqOsMmpNsojWAqgM7s6DCUxNAAASl0lEQVTEiRP56U9/eti6pUuXMnfuXCZOnMj5559f+D/z\naFJSUgq/RpqUlETdunULn+flBX2xd911F8cffzzLly8nPz+ftLS0wv0/++wzkpKS+OKLL8jPzycp\nqfSeyZK+rurunHfeeTz55JNR9ynr664FcScnJxfGLVJpTjo9eGzbCAunQtYMWDYNOg4KksbJZ9X6\nO8NVs4izRo0asWNHMGXkBRdcwCOPPFLYp//ZZ5/x5ZdfsmXLFurXr8/IkSO59dZb+fDDDw/btyK+\n+eYbWrZsSVJSEtOmTePgwYNA0PUzevRonnjiCTp37szkyZNLPUb//v2ZMWMGAKtWrWLFimB6kb59\n+/L++++zYcMGAHbv3l2k5fLUU08V/lvQ4ojVzyUSc8WL4TlL4NEh8PA5sOpftboYXmtaFonSrFkz\nzjrrLE499VQGDx7MVVddVfih2bBhQ6ZPn86GDRsYP348SUlJpKSk8MADDwAwZswYBg8eTMuWLXnz\nzTePOoYbbriBSy+9lFmzZjFw4EAaNGgAwP/93//Rr18/+vXrR3p6Or1792bIkCF07tz5sGNcf/31\njB49mu7du5Oenk6fPn0AaNGiBY8++igjRoxg3759ANx555107NgRgH379nH66aeTn59f2PoYPnw4\n1113HVOmTGH27NlH/XOJxE1BMfysmyDrCVh4H8weDY1PhjNuhPSroW7DREdZqWr02FBr1qwp8YNP\nKkfbtm3JzMwss0BfEfr9SqXJPwjr5gb1jM2LIa1xjSmGa2woEZFYSUqGzj+An8yDH88LRrd9dzLc\nfSo8PxZy1yU6wrhTN1Q1cfrppxd28xSYNm0a3bp1i+l5Xn31VSZMmFBkWbt27Xj22WeP+Fj6VpPU\nSCedDifNCIvh9wXdVLWgGB7XbigzGwTcAyQDf3f3P5Sy3WXALKC3u2eGy7oDfwOOAfLDdaUOM6pu\nqNpHv1+pEnZtjRgmfRuc2DNIGp2HVYth0hPeDWVmycBUYDDQBRhhZl1K2K4RMA5YHLGsDjAd+Jm7\ndwUGAAfiFauIyFErKIbfvAqGTIa9XwfF8Ht7wuK/wf5diY4wJuJZs+gDbHD3Te6+H5gJXFTCdr8F\n/gREthrOB1a4+3IAd9/m7gfjGKuISMWk1g+K3mMzgwEMGx4PL/8PTO4Cr/8WdnyR6AgrJJ7JohWw\nOeJ1TriskJn1ANq4+4vF9u0IuJm9amYfmtn/xDFOEZHYKSiGXzs/ohj+16AYPufn1bYYHs8OtZIq\nPIUFEjNLAu4CRpWwXR3ge0BvYDfwetiv9nqRE5iNAcYAnHTSSbGJWkQkVgqK4Vs3wKKpQTH8w8eh\n4+CwGH5mtSmGx7NlkQO0iXjdGtgS8boRcCrwlpl9AvQF5phZRrjv2+6+1d13A3OBnsVP4O4PuXuG\nu2e0aNEiTj9GxRztEOUXXnghX3/9dRwiio/f/OY3/OUvfzmqfbOyspg7d25MjiVSJTU/JRjp9uZV\ncPZtwb0aj14Y3Bm++tlqcWd4PJPFEqCDmbUzs1RgODCnYKW7f+Puzd29rbu3BRYBw8JvQ70KdDez\n+mGx+2ygWk6AUFqyKBhyozRz586lcePG8QqrSimeLERqrIYtYODEYDiRIX8NiuGzRoXF8IeqdDE8\nbt1Q7p5nZmMJPviTgUfcfbWZTQIy3X1OGft+ZWaTCRKOA3Pd/aUKBfTybfCflRU6xGFO6AaDS/w2\ncKHI+SxSUlJo2LAhLVu2JCsri+zsbC6++GI2b97M3r17uemmmxgzZgxw6O7nnTt3MnjwYL73ve+x\nYMECWrVqxfPPP0+9evVKPN+AAQPo0aMHS5cuJTc3l8cff5zf//73rFy5kiuvvJI777wToMTz/vvf\n/+bcc89l4cKFNG3alLPPPpvbb7+d888/v8Rz/e53v+Pxxx+nTZs2tGjRgl69egGwceNGbrzxRnJz\nc6lfvz4PP/ww3/3udxk1ahRpaWmsXr2aL774gsmTJxcOmrhnzx7ee+89Jk6cCEB2djYDBgzg008/\n5eabby4c8VakRkitD72vhV6jYe1LwTDpL4+HN38XLD/9p9DwuERHWURcvwTs7nMJupAil5U4nKq7\nDyj2ejrB12ertcj5LN566y2GDBnCqlWraNeuHQCPPPIITZs2Zc+ePfTu3ZtLL72UZs2aFTnG+vXr\nefLJJ3n44Ye54ooreOaZZxg5cmSp50xNTeWdd97hnnvu4aKLLmLp0qU0bdqU73znO9xyyy00a9as\nxPOefPLJTJgwgZ/97GecfvrpdOnSpdREsXTpUmbOnMmyZcvIy8ujZ8+ehclizJgxPPjgg3To0IHF\nixdzww038MYbbwDBjXpvv/02GzduZODAgWzYsIFJkyaRmZnJfffdBwTdUGvXruXNN99kx44ddOrU\nieuvv56UlJQK/z5EqpSkZOgyLHh8ujhIGu/+NRhW5LQr4YyfQ4uOiY4SqE13cEdpAVSWPn36FCYK\ngClTphTeHb1582bWr19/WLJo164d6enpQPnmexg2bBgQzG/RtWtXWrZsCUD79u3ZvHkzzZo1K/W8\n1157LbNmzeLBBx8kKyur1HO8++67XHLJJdSvX7/IOXfu3MmCBQu4/PLLC7eNvPP8iiuuICkpiQ4d\nOtC+fXvWrl1b4vGHDBlC3bp1qVu3LscddxxffPEFrVu3LvPnFqnWIovhC++D5U9WqWJ47UkWVUTB\niK8Ab731Fq+99hoLFy6kfv36DBgwgL17D79JvWCuBwjme4ic0rQkkXNaRO5bMMdFWefdvXs3OTk5\nQPDB36hRo1LPU9L8FPn5+TRu3LjURFN8n9LmuCj+M2uOC6k1mp8CP7gbBv7y0J3hj74MrXodujM8\nKbnSw9JAgnFW1twN33zzDU2aNKF+/fqsXbuWRYsWVUpMZZ13woQJXH311UyaNInrrruu1GP079+f\nZ599lj179rBjxw5eeOEFIJhGtV27dsyaNQsIJkdavnx54X6zZs0iPz+fjRs3smnTJjp16qT5LURK\nUrwYvnt7UAyf0iMhxXAliziLnM9i/PjxRdYNGjSIvLw8unfvzu23307fvn0rJabSzvv222+zZMmS\nwoSRmprKP//5zxKP0bNnT6688krS09O59NJLi0wFO2PGDP7xj39w2mmn0bVrV55//vnCdZ06deLs\ns89m8ODBPPjgg6SlpTFw4ECys7NJT08vnCxJREIFxfCfL4UrpgWF75fHw11d4Y07YeeXlRKG5rOQ\nSjNq1CiGDh3KZZddFpPj6fcrtdani4Ii+NqXIDkVev4ILvzLUdU0yjuQoGoWIiLVzUl9g8fW9cGc\n4fl5cS9+K1lUUzfeeCPvv/9+kWU33XQTo0ePjul5tm3bxve///3Dlr/++uuHfWsrmkcffTRGUYkI\nAM07BMXwSughUrKopqZOnVop52nWrFmZX6EVkSqgEr5SW+ML3DWlJiNF6fcqUrlqdLJIS0tj27Zt\n+mCpYdydbdu2kZaWluhQRGqNGt0N1bp1a3JycsjNzU10KBJjaWlpuqNbpBLV6GSRkpJSZGgNERE5\nOjW6G0pERGJDyUJERKJSshARkahqzHAfZpYL/LsCh2gObI1ROLGkuI6M4joyiuvI1MS4Tnb3qPNS\n15hkUVFmllme8VEqm+I6MorryCiuI1Ob41I3lIiIRKVkISIiUSlZHPJQogMoheI6MorryCiuI1Nr\n41LNQkREolLLQkREoqoVycLMks1smZm9WMK6umb2lJltMLPFZtY2Yt3EcPk6M7ugkuP6hZllm9kK\nM3vdzE6OWHfQzLLCx5xKjmuUmeVGnP/aiHXXmNn68HFNJcd1V0RMH5nZ1xHr4n29PjGzleHxM0tY\nb2Y2JXwvrTCznhHr4nbNyhHX1WE8K8xsgZmdVt594xzXADP7JuJ39quIdYPCv8cNZnZbJcc1PiKm\nVeH7qml59q1gXI3NbLaZrTWzNWZ2RrH1lfP+cvca/wB+ATwBvFjCuhuAB8Pnw4GnwuddgOVAXaAd\nsBFIrsS4BgL1w+fXF8QVvt6ZwOs1CrivhOVNgU3hv03C500qK65i2/0ceKQSr9cnQPMy1l8IvAwY\n0BdYXBnXrBxxnVlwPmBwQVzl2TfOcQ0o5b2XHP4dtgdSw7/PLpUVV7FtfwC8UUnX6zHg2vB5KtA4\nEe+vGt+yMLPWwBDg76VschHBLwNgNvB9M7Nw+Ux33+fuHwMbgD6VFZe7v+nuu8OXi4BKGWK1HNer\nNBcA8919u7t/BcwHBiUorhHAk7E6dwxcBDzugUVAYzNrSZyvWTTuviA8L1Tie6wC+gAb3H2Tu+8H\nZhJc20SolPeYmR0D9Af+AeDu+93962KbVcr7q8YnC+Bu4H+A/FLWtwI2A7h7HvAN0CxyeSgnXFZZ\ncUX6CcH/HAqkmVmmmS0ys4tjGFN547o0bO7ONrM24bIqcb3C7rp2wBsRi+N5vQAcmGdmS81sTAnr\nS7s28b5m0eKKVPw9diT7xiOuM8xsuZm9bGZdw2VV4nqZWX2CD91njnTfo9AeyAX+GXbB/t3MGhTb\nplLeXzV6iHIzGwp86e5LzWxAaZuVsMzLWF5ZcRVsOxLIAM6OWHySu28xs/bAG2a20t03VlJcLwBP\nuvs+M/sZQavsHKrI9SLoSpzt7gcjlsXlekU4Kzz+ccB8M1vr7u9E/ggl7BPX91g54wqCMxtIkCy+\nd6T7ximuDwmGoNhpZhcCzwEdqCLXi6AL6n13334U+x6pOkBP4OfuvtjM7gFuA26P2KZS3l81vWVx\nFjDMzD4haLKeY2bTi22TA7QBMLM6wLHA9sjlodbAlkqMCzM7F/glMMzd9xUsd/ct4b+bgLeAHpUV\nl7tvi4jlYaBX+Dzh1ys0nGLdA3G8XsWP/yXwLId3V5Z2beJ5zcoTF2bWnaBr7yJ333Yk+8YrLnf/\n1t13hs/nAilm1pwqcL1CZb3HYn29coAcd18cvp5NkDyKbxP/91c8CjJV8UHpRbMbKVrgfjp83pWi\nBe5NxLjAHSWuHgTFvA7FljcB6obPmwPriWGRrxxxtYx4fgmwKHzeFPg4jK9J+LxpZcUVrutEUGi0\nyrpeQAOgUcTzBcCgYtsMoWgB8oN4X7NyxnUSQS3uzCPdN85xnVDwOyT40P00vHZ1wr/DdhwqcHet\nrLjCdQX/mWxQGdcrPOa7QKfw+W+APyfi/VWju6FKY2aTgEx3n0NQOJpmZhsI3gTDAdx9tZk9DWQD\necCNXrRrI95x/RloCMwK6u186u7DgM7A38wsn6Bl+Ad3z67EuMaZ2TCCa7Kd4NtRuPt2M/stsCTc\nbZIXbabHOy4Iio4zPfxLCcX7eh0PPBv+juoAT7j7K2EXHe7+IDCX4BsrG4DdwOhwXTyvWXni+hVB\nfe7+cLs8DwajK3HfSozrMuB6M8sD9gDDw99pnpmNBV4l+GbUI+6+uhLjguA/SPPcfVe0fWMUFwTf\n7pthZqkEyXJ0It5fuoNbRESiquk1CxERiQElCxERiUrJQkREolKyEBGRqJQsREQkKiULERGJSslC\npJJZMJx186Pcd5SZnRiLY4kcCSULkeplFHBitI1EYk3JQmotM2trwYQyf7dgMpsZZnaumb0fThbT\nJ3wsCEf8XGBmncJ9f2Fmj4TPu4X71y/lPM3MbF54jL8RMcCbmY00sw8smDTnb2aWHC7faWZ/NbMP\nLZj8qoWZXUYwqOSMcPt64WF+Hm630sy+G89rJrWXkoXUdqcA9wDdge8CVxGMvnor8P+AtUB/d+9B\nMDzG/4X73Q2cYmaXAP8EfuqH5h8p7tfAe+Ex5hCMyYSZdQauJBixNB04CFwd7tMA+NDdewJvA792\n99lAJnC1u6e7+55w263hdg+EcYvEXK0cG0okwsfuvhLAzFYDr7u7m9lKoC3BwHGPmVkHguGdUwDc\nPd/MRgErgL+5+/tlnKM/8MNwv5fMrGDCoe8TjNq7JBxXqB7wZbguH3gqfD4d+FcZxy9Yt7TgPCKx\npmQhtd2+iOf5Ea/zCf4+fgu86e6XWDA/+1sR23cAdlK+GkJJg7AZ8Ji7TzzK/QsUxHwQ/U1LnKgb\nSqRsxwKfhc9HFSw0s2MJuq/6A83CekJp3iHsXjKzwQTDRQO8DlwWTpiDmTW1YKY/CP42C455FfBe\n+HwH0KgCP4/IUVGyECnbn4Dfm9n7BMNiF7gLuN/dPyKYZe4PBR/6JbgD6G9mHwLnE8zPQDhU+v8S\nTMe5gmCO5JbhPruArma2lGAmwknh8keBB4sVuEXiTkOUi1RBZrbT3RsmOg6RAmpZiIhIVGpZiMSI\nmY0Gbiq2+H13vzER8YjEkpKFiIhEpW4oERGJSslCRESiUrIQEZGolCxERCQqJQsREYnq/wOBQfEi\n1BIsRQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xc0b62e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 将GridSerachCV得到的结果反映在图示中\n",
    "#打印最佳参数与最佳性能得分\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "#将结果保存在csv文件中\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('Preds_for_max_depth.csv')\n",
    "\n",
    "\n",
    "pyplot.plot(max_depth, -test_means, label= 'test_max_depth')\n",
    "pyplot.plot(max_depth, -train_means, label= 'train_max_depth')\n",
    "\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_Logloss.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从图可知，随max_depth增大，以mlogloss为评价指标时，train数据集的性能一直在提高，而test数据集的性能在max_depth=5左右时最佳。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
